'''
process the orignal data
'''
import codecs
import re
import os
import csv

def zero_digits(s):
    """
    Replace every digit in a string by a zero.
    """
    return re.sub('\d', '0', s)

# 返回 句子+标签的字符串
def sentence_list2str(sentence):
    str_s = ''
    str_t = ''
    for sent in sentence:
        str_s = str_s + sent[0]
        str_t = str_t + sent[1] + ' '
    assert len(str_s) == len(str_t.strip().split())
    str_sentence = str_s.replace(' ', '') + '%%' + str_t.strip() + '\n'
    return str_sentence

# 以list形式返回所有句子
def load_sentences(path, zeros):
    """
    Load sentences. A line must contain at least a word and its tag.
    Sentences are separated by empty lines.
    """
    sentences = []
    sentence = []
    num = 0
    csvfile = open('csvfile.csv', 'a')
    csvw= csv.writer(csvfile)
    for line in codecs.open(path, 'r', 'utf8'):
        num+=1
        line = zero_digits(line.rstrip()) if zeros else line.rstrip()
        # print(list(line))
        if not line:
            if len(sentence) > 0:
                if 'DOCSTART' not in sentence[0][0]:
                    sentences.append(sentence_list2str(sentence))
                    csvw.writerow(sentence_list2str(sentence).split('%%'))
                sentence = []
        else:
            if line[0] == " ":
                line = "$" + line[1:]
                word = line.split()
                # word[0] = " "
            else:
                word = line.split()
            assert len(word) >= 2, print([word[0]])
            sentence.append(word)

    if len(sentence) > 0:
        if 'DOCSTART' not in sentence[0][0]:
            sentences.append(sentence)
    return sentences

# 将句子保存在文件中
def write2file(file_name, lines):
    print('生成文件：', file_name)
    file = open(file_name, 'w')
    file.writelines(lines)
    file.close()


# 将word-tag形式的数据，转换成 words tags 的形式，并保存在文件中
def process_data(data_name):
    print('正在处理文件：', data_name)
    name = data_name.split('.')[0]
    sentences = load_sentences(os.path.join('data', data_name), zeros=False)
    print('处理%s条数据'%len(sentences))
    write2file(os.path.join('data', name+'_lines.txt'), sentences)




# if __name__ == '__main__':
#     process_data('valid_all.txt')
